import re
import time
import random
import requests
import urllib.parse
from html.parser import HTMLParser
from pymysql.converters import escape_string
from lxml import html

etree = html.etree
from mysql.main import python_sql_mysql
from redis_con.detail_proxy import my_get_proxy
from crawler_bk.xpath_bk_detail import xpath_detail
from crawler_baidu.logger import LOG

# 浏览器的ua
headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Cache-Control': 'max-age=0',
    'Connection': 'keep-alive',
    'Cookie': 'BAIKE_SHITONG=%7B%22data%22%3A%22c9f839443608c6ae19a215a99e9301b956181bdefb99f20edd9046db95c75bffe7e54c800546fd43daebf3f948e4cbb4323c857f14b139a9a91ec07bdf95d6a9b48ad11f2a352970a93bcb409d0a91a544c369aa50d4c4ad9c6d468f8f658152517aabbd181f1137ebfcd8b1b99fd3c5787f28f44efd4c227b42e9aca923af90%22%2C%22key_id%22%3A%2210%22%2C%22sign%22%3A%22aca581b4%22%7D; BIDUPSID=11C80C6A3845144B87C9D85CE5EE5297; PSTM=1635904619; BAIDUID=11C80C6A3845144B09A6B9EEA0E98F0D:FG=1; __yjs_duid=1_784aded915063e8557abb21970749c2b1635928884800; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; BDUSS=1GSTdPU0w4ZWZ1M3ZyTS1uRE0zdmNaUmxJenVpa35QY1hOVndVcklJVVRRZFpoSVFBQUFBJCQAAAAAAAAAAAEAAABhcBqCs~W8-8TjX2kwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABO0rmETtK5hZ0; BDUSS_BFESS=1GSTdPU0w4ZWZ1M3ZyTS1uRE0zdmNaUmxJenVpa35QY1hOVndVcklJVVRRZFpoSVFBQUFBJCQAAAAAAAAAAAEAAABhcBqCs~W8-8TjX2kwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABO0rmETtK5hZ0; delPer=0; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; RT="z=1&dm=baidu.com&si=1916lxwm81t&ss=kwvzfi98&sl=2&tt=qy&bcn=https%3A%2F%2Ffclog.baidu.com%2Flog%2Fweirwood%3Ftype%3Dperf&ld=2c1&ul=2f4&hd=2fa"; MCITY=-179%3A; BCLID=8538340688079721998; BDSFRCVID=-8-OJeC62mq0MTRH45j5KGJy1f6PVKQTH6aIQ7AQA6C1I615t01AEG0PEM8g0Kubq_zzogKKKgOTHICF_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF=tbkD_C-MfIvDqTrP-trf5DCShUFs-45CB2Q-XPoO3KtMSRTkyMJKLjDUQP68J4QiWbRM2MbgylRp8P3y0bb2DUA1y4vpKMP8bmTxoUJ2XMKVDq5mqfCWMR-ebPRiB-b9QgbA5hQ7tt5W8ncFbT7l5hKpbt-q0x-jLTnhVn0MBCK0hI0ljj82e5PVKgTa54cbb4o2WbCQH45M8pcN2b5oQTO33xrXBUbH0GbiKhcXtR5vOIJTXpOUWfAkXpJvQnJjt2JxaqRC5hkBfq5jDh3MBpQDhtoJexIO2jvy0hvcWb3cShnVLUjrDRLbXU6BK5vPbNcZ0l8K3l02V-bIe-t2XjQhDNtDt60jfn3aQ5rtKRTffjrnhPF3yx4FXP6-hnjy3b4J-JT8Wp3KEqcoQU6BDtLUyN3MWh3RymJ42-39LPO2hpRjyxv4bUn-5toxJpOJXaILWl52HlFWj43vbURvyP-g3-7A3M5dtjTO2bc_5KnlfMQ_bf--QfbQ0hOhqP-jBRIEoDtKJIP2MI_rK4ofMJD8-p8Xb-cLKDCXLnjOKJjoKROvhjRj3f0gyxomtjjHBgOk_UbdJI5UOqvjyRuaeMuUQH7yLUkqKCOqsRjnyC02OfI6Lx6AMb30QttjQUnufIkja-5tWJ6tqb7TyURvbU47y-rm0q4Hb6b9BJcjfU5MSlcNLTjpQT8r5MDOK5OuJRLDVCIyJK8BMCv4btOEK4LS52Tea4raK4oJ3R7tKRA_Hn7zeIFWqf4pbt-qJtrDtgjnbqP2JRFhOp3_2jQtjl8ZynQnBT5KWGCf5xjpJq8aODLljJ82Df_kQN3T0xuO5bRiLRohX-c0Dn3oyURJXp0nytbly5jtMgOBBJ0yQ4b4OR5JjxonDh83bG7MJUutfD7H3KChfCKKbfK; BCLID_BFESS=8538340688079721998; BDSFRCVID_BFESS=-8-OJeC62mq0MTRH45j5KGJy1f6PVKQTH6aIQ7AQA6C1I615t01AEG0PEM8g0Kubq_zzogKKKgOTHICF_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF_BFESS=tbkD_C-MfIvDqTrP-trf5DCShUFs-45CB2Q-XPoO3KtMSRTkyMJKLjDUQP68J4QiWbRM2MbgylRp8P3y0bb2DUA1y4vpKMP8bmTxoUJ2XMKVDq5mqfCWMR-ebPRiB-b9QgbA5hQ7tt5W8ncFbT7l5hKpbt-q0x-jLTnhVn0MBCK0hI0ljj82e5PVKgTa54cbb4o2WbCQH45M8pcN2b5oQTO33xrXBUbH0GbiKhcXtR5vOIJTXpOUWfAkXpJvQnJjt2JxaqRC5hkBfq5jDh3MBpQDhtoJexIO2jvy0hvcWb3cShnVLUjrDRLbXU6BK5vPbNcZ0l8K3l02V-bIe-t2XjQhDNtDt60jfn3aQ5rtKRTffjrnhPF3yx4FXP6-hnjy3b4J-JT8Wp3KEqcoQU6BDtLUyN3MWh3RymJ42-39LPO2hpRjyxv4bUn-5toxJpOJXaILWl52HlFWj43vbURvyP-g3-7A3M5dtjTO2bc_5KnlfMQ_bf--QfbQ0hOhqP-jBRIEoDtKJIP2MI_rK4ofMJD8-p8Xb-cLKDCXLnjOKJjoKROvhjRj3f0gyxomtjjHBgOk_UbdJI5UOqvjyRuaeMuUQH7yLUkqKCOqsRjnyC02OfI6Lx6AMb30QttjQUnufIkja-5tWJ6tqb7TyURvbU47y-rm0q4Hb6b9BJcjfU5MSlcNLTjpQT8r5MDOK5OuJRLDVCIyJK8BMCv4btOEK4LS52Tea4raK4oJ3R7tKRA_Hn7zeIFWqf4pbt-qJtrDtgjnbqP2JRFhOp3_2jQtjl8ZynQnBT5KWGCf5xjpJq8aODLljJ82Df_kQN3T0xuO5bRiLRohX-c0Dn3oyURJXp0nytbly5jtMgOBBJ0yQ4b4OR5JjxonDh83bG7MJUutfD7H3KChfCKKbfK; BAIDUID_BFESS=F4CDF0A4046F805E7230668DB636974B:FG=1; Hmery-Time=1921904055; PSINO=3; __yjs_st=2_NGNhOTRlZDczZGVhYWIxNjBjNjMxYTAyNzIxMTgxZTQ5Zjc4NjE5YWZhMjBlNDBjZjEwYTMwNmI0NWFkZGE5MGYzN2VlZTY1ZGNkZDVhYWE1Nzc0Y2UxYzUxMjcyMDc2Y2YwOGMwMTQzODUwMTQ1Y2I4YTRiNmE1MzE4OTNjZjE1ZDI1ZDcyNDkzYjZlZjc0MmViMzFhMDJkZjI1NTYxZGNlMDMwYmZlMThhYzVjZGY4N2JhYjkyYmI5MzkzYWE3NWJjOGNjZWRhMDU1ZjU0YWZkZDBjZmI3MTRlYWU3MmE3ZThmZjYxYWQ5Nzk1YmFkMjkwNDIyOGI1ZDllNmZkZV83XzdiZDFiMGY3; H_PS_PSSID=35106_31253_35240_34584_34505_35245_34606_35315_26350_35210_22157; BA_HECTOR=812l0h0g818h0l24a91gr2okr0r; zhishiTopicRequestTime=1639015068744; Hm_lvt_55b574651fcae74b0a9f1cf9c8d7c93a=1636706737,1637048328,1638428882,1639015072; baikeVisitId=3944edfa-a7dd-4854-ac10-e3a180cdeb14; BK_SEARCHLOG=%7B%22key%22%3A%5B%22%E5%A4%8F%E6%99%AE%22%2C%2223%E5%8C%BA%22%2C%22%E5%8D%83%E8%B6%B3%22%2C%22%E7%B1%B3%E6%8A%80%22%5D%7D; Hm_lpvt_55b574651fcae74b0a9f1cf9c8d7c93a=1639015154; ab_sr=1.0.1_NGQxYTgxMzcxNjMwNjQxMTc1MmYyYjQxNjI0Nzk2ZTk2YmRiOTc1ZDNmN2M3YWQ0NmI4ODRlZTBkZGI3OGY0OGEyYmU0ZTdjOTU3NGZiNGE4OTBiY2FjODRkOTQxMjI3YzE5NjI2NDcxNDNkNDMyYmNhNmViMGExZDY0ZDJmMjAzNGNmYmYwMDEyNzBkOTgwMWY3ZmU4MzdjNjc5MWM0ZGQyOTczZDUxZjUyMWE4YmQ4YTViNGQ3MzljM2JkZjZm',
    'Host': 'baike.baidu.com',
    'Referer': 'https://baike.baidu.com/search?word=udx&pn=0&rn=0&enc=utf8',
    'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="96", "Google Chrome";v="96"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Windows"',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'same-origin',
    'Sec-Fetch-User': '?1',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36',
}


def insert_bk_brand_url(tb_shop_id, tb_shop_name, search_name, raw_html):
    """
    处理搜索词条直接出来搜索全站的操作
    :param tb_shop_id: 解百的品牌id
    :param tb_shop_name: 解百的品牌名称
    :param search_name: 解百品牌在百度百科的搜索名称
    :param raw_html: 原始的html页面
    :return: 不返回
    """
    if not python_sql_mysql(db_name=db_name, is_return=True,
                            sql="select * from bk_search_raw_html "
                                "where tb_shop_id=%s" % (
                                        tb_shop_id)):
        LOG.info('进入插入方法')
        python_sql_mysql(db_name=db_name, is_return=False,
                         sql="insert into bk_search_raw_html "
                             "(tb_shop_id,tb_shop_name,search_name,raw_html) "
                             "VALUES (%s,\'%s\',\'%s\',\'%s\')" % (
                                 tb_shop_id, escape_string(tb_shop_name),
                                 escape_string(search_name),
                                 escape_string(raw_html)))
    html1 = etree.HTML(raw_html)
    for i, j in enumerate(
            html1.xpath('//*[@id="body_wrapper"]/div[1]/dl/dd/a')):
        if i == 0:
            brand_name = html1.xpath(
                '//*[@id="body_wrapper"]/div[1]/dl/dd[{}]/a'.format(i + 1))
            brand_name = html.tostring(brand_name[0])
            brand_name = HTMLParser().unescape(brand_name.decode('utf-8'))
            pattern = re.compile(r'>(.*?)<')
            brand_name_list = pattern.findall(brand_name)
            brand_name = ''.join(brand_name_list)
            brand_url = ''.join(html1.xpath(
                '//*[@id="body_wrapper"]/div[1]/dl/dd[{}]/a/@href'.format(
                    i + 1)))
            LOG.info('brand_url:%s' % (brand_url))
            if brand_url.startswith('http'):
                pass
            else:
                brand_url = 'https://baike.baidu.com' + brand_url
            if 'https' in brand_url:
                pass
            else:
                brand_url.replace('http', 'https')
            LOG.info('品牌在内部')
            LOG.info('%s' % (brand_url))
            if not python_sql_mysql(db_name=db_name, is_return=True,
                                    sql="select * from bk_brand_url"
                                        " where  tb_shop_id=%s" % (
                                                tb_shop_id)):
                tb_shop_id = tb_shop_id
                tb_shop_name = tb_shop_name
                search_name = search_name
                # 将此品牌写入mysql
                LOG.info('插入数据库')
                python_sql_mysql(
                    sql="insert into bk_brand_url (tb_shop_id," \
                        "tb_shop_name,search_name,bk_brand_name,content_url) " \
                        "VALUES (%s,\'%s\',\'%s\',\'%s\',\'%s\')" % \
                        (
                            tb_shop_id,
                            escape_string(tb_shop_name),
                            escape_string(search_name),
                            escape_string(brand_name),
                            escape_string(brand_url)
                        ), is_return=False,
                    db_name=db_name)
                search_entry(tb_shop_id=tb_shop_id,
                             tb_shop_name=tb_shop_name,
                             url=brand_url,
                             search_name=search_name)
            else:
                LOG.info('通过全站搜索进入词条')
                search_entry(tb_shop_id=tb_shop_id,
                             tb_shop_name=tb_shop_name,
                             url=brand_url,
                             search_name=search_name)


def search_entry(tb_shop_id, tb_shop_name, search_name, url):
    """
    搜索百度词条,存在三种情况
    1.词条存在:直接进入词条信息,处理相关信息直接进行保存
    2.词条套着词条:词条一样,但是官方给出的解释不一样(就会出现一次词条两种解释) 铁兄弟
    2.词条不存在,但是存在类似词条,例如全站搜索类似的,直接进行处理
    3.词条不存在,也没有类似词条,直接放弃对该词条的操作
    :param jb_brand_id: 解百的品牌id
    :param jb_brand_name: 解百的品牌名称
    :return: 不返回
    """
    if not python_sql_mysql(
            sql="select * from bk_detail_raw_html where tb_shop_id=%s " % (
                    tb_shop_id), db_name=db_name,
            is_return=True) and not python_sql_mysql(
        sql="select * from error_bk_intr_detail where tb_shop_id=%s" % (
                tb_shop_id), db_name=db_name, is_return=True):

        response = requests.get(url=url, headers=headers,
                                proxies=my_get_proxy())
        LOG.info('%s' % (response.status_code))
        response.encoding = 'utf-8'
        raw_html = response.text
        html1 = etree.HTML(raw_html)
        if '您所访问的页面不存在...' in response.text:
            LOG.warning('页面不存在')
            return False
        # elif html1.xpath('//ul[contains(@class,"custom_dot")]'):
        elif html1.xpath('//div[@class="lemmaWgt-subLemmaListTitle"]'):
            LOG.info('词条种嵌有词条')
            return False
        elif html1.xpath('//a[@class="result-title"]'):
            LOG.info('直接进入了全站搜索')
            insert_bk_brand_url(tb_shop_id=tb_shop_id,
                                tb_shop_name=tb_shop_name,
                                search_name=search_name, raw_html=raw_html)
        else:
            LOG.info('进入了词条操作')
            python_sql_mysql(db_name=db_name,
                             sql="insert into bk_detail_raw_html "
                                 "(tb_shop_id,tb_shop_name,raw_html) "
                                 "VALUES (%s,\'%s\',\'%s\')" % (
                                     tb_shop_id,
                                     escape_string(tb_shop_name),
                                     escape_string(raw_html)),
                             is_return=False)
            LOG.info('插入成功')
            if not python_sql_mysql(
                    sql="select * from bk_intr_detail where  tb_shop_id=%s" % (
                            tb_shop_id), db_name=db_name,
                    is_return=True):
                bk_brand_name, bk_introduce_one, bk_introduce_json_one, \
                bk_introduce_json_two = xpath_detail(raw_html)

                insert_sql = "insert into bk_intr_detail (tb_shop_id,tb_shop_name,search_name," \
                             "bk_brand_name,bk_introduce_one,bk_introduce_dict_one," \
                             "bk_introduce_dict_two) VALUES (%s,\'%s\',\'%s\',\'%s\',\'%s\',\'%s\',\'%s\')" % (
                                 tb_shop_id,
                                 escape_string(tb_shop_name),
                                 escape_string(search_name),
                                 escape_string(bk_brand_name),
                                 escape_string(bk_introduce_one),
                                 escape_string(bk_introduce_json_one),
                                 escape_string(bk_introduce_json_two))
                LOG.info('%s' % (insert_sql))
                if '\'\\"NULL\\"\'' in insert_sql:
                    LOG.info('在里面')
                insert_sql = insert_sql.replace('\'\\"NULL\\"\'',
                                                'NULL').replace(
                    '\'NULL\'', 'NULL')
                LOG.info('%s' % (insert_sql))
                python_sql_mysql(db_name=db_name,
                                 sql=insert_sql,
                                 is_return=False)
                LOG.info('插入成功')
                time.sleep(random.randint(4, 6))
                return True


def main():
    """
    主函数
    :return: 不返回
    """
    datas = python_sql_mysql(db_name=db_name, is_return=True,
                             sql="SELECT tb_shop_id,tb_shop_name "
                                 "FROM shop_detail where is_tianmao=1")
    exis_data_1 = [i[0] for i in
                   python_sql_mysql(db_name=db_name, is_return=True,
                                    sql="select tb_shop_id from bk_intr_detail")]
    exis_data = [i[0] for i in
                 python_sql_mysql(
                     sql="select tb_shop_id from error_bk_intr_detail",
                     db_name=db_name, is_return=True)]
    LOG.info('%s', datas)
    for data in datas:
        tb_shop_id = data[0]
        tb_shop_name = data[1]
        if tb_shop_id not in exis_data_1 and tb_shop_id not in exis_data:
            # 判断这个词条没有被遍历过
            if 'outlets' in data[1]:
                continue
            elif '官方旗舰店' in data[1]:
                search_name = data[1].split('官方旗舰店')[0]
            elif '旗舰店' in data[1]:
                search_name = data[1].split('旗舰店')[0]
            elif '官方旗舰' in data[1]:
                search_name = data[1].split('官方旗舰')[0]
            elif '官方网店' in data[1]:
                search_name = data[1].split('官方网店')[0]
            else:
                continue
            LOG.info('%s %s' % (search_name, tb_shop_id))
            url = 'https://baike.baidu.com/item/{}'.format(
                urllib.parse.quote(search_name))
            if search_entry(tb_shop_id=tb_shop_id, tb_shop_name=tb_shop_name,
                            search_name=search_name, url=url):
                # 通过直接进入词条直接获得了爬虫信息
                pass
            else:
                LOG.warning('直接搜索词条错误')
                pattern = re.compile("[A-Za-z]")
                if re.findall(pattern, search_name):
                    pass
                else:
                    search_name = search_name + '品牌'
                url = 'https://baike.baidu.com/search/none?word={}'.format(
                    urllib.parse.quote(search_name))
                response = requests.get(url=url, headers=headers,
                                        proxies=my_get_proxy())
                LOG.info('%s' % (response.status_code))
                response.encoding = 'utf-8'
                raw_html = response.text
                html1 = etree.HTML(raw_html)
                if html1.xpath('//a[@class="result-title"]'):
                    LOG.info('进入insert_bk_brand_url方法')
                    insert_bk_brand_url(search_name=search_name,
                                        tb_shop_id=tb_shop_id,
                                        tb_shop_name=tb_shop_name,
                                        raw_html=raw_html)
                else:
                    # 如果通过进入词条或者是全站搜索都无法搜索到词条信息,认定为网络上没有相关信息
                    if not python_sql_mysql(
                            sql="select * from error_bk_intr_detail where tb_shop_id=%s" % (
                                    tb_shop_id), is_return=True,
                            db_name=db_name):
                        python_sql_mysql(db_name=db_name,
                                         sql="insert into error_bk_intr_detail "
                                             "(tb_shop_id,search_name,tb_shop_name) "
                                             "VALUES (%s,\'%s\',\'%s\')" % (
                                                 tb_shop_id,
                                                 escape_string(search_name),
                                                 escape_string(tb_shop_name)),
                                         is_return=False)
                        LOG.info("%s 插入error_bk_intr_detail表" % (tb_shop_id))
                        time.sleep(3)


if __name__ == '__main__':
    db_name = 'crawler_bk'
    main()
